from scrapy_redis.spiders import RedisSpider
from kuaidaili_redis.items import KuaidailiRedisItem


class MyRedisSpider(RedisSpider):
    """Spider that reads urls from redis queue (myspider:start_urls)."""
    name = 'myredisspider'
    redis_key = 'myredisspider:start_urls'

    def __init__(self, *args, **kwargs):
        # Dynamically define the allowed domains list.
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(MyRedisSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        print('开始爬取...')
        ip_list = response.xpath('//tbody/tr/td[1]/text()').extract()
        port_list = response.xpath('//tbody/tr/td[2]/text()').extract()
        for i in range(len(ip_list)):
            item = KuaidailiRedisItem()
            item['ip'] = ip_list[i]
            item['port'] = port_list[i]
            yield item
